# -*- coding: utf-8 -*-

'''
该脚本从国务院新闻发布平台搜索关于国内假期安排的通知，下载处理后生成国内法定节假日安排，是第一手的权威数据
目前只处理了初始放假安排的通知，没有处理变动假期安排的通知，假如假期产生了变动，只能做到通知维护人，由维护人手动处理
'''

import urllib.request
from urllib.parse import quote
import random
import re
import json
import sys
import time
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from pymysql import *
from dateutil.relativedelta import relativedelta

''' 
#@获取403禁止访问的网页 
'''
my_headers = [
	"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0"
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)"]

config = {
	'user': 'root',
	'password': 'xxx',
	'host': 'xxx',
	'port': 3306,
	'database': 'xxx',
	'charset': 'utf8'
}


def getContent(url, host, referer):
	req = urllib.request.Request(url)
	req.add_header("User-Agent", random.choice(my_headers))
	req.add_header("Host", host)
	req.add_header("Referer", referer)
	req.add_header("GET", url)
	return urllib.request.urlopen(req).read()


def commonHoliday(year, bs):
	print('**************** year:', year, '****************')
	finalTexts = []
	for p in bs.find_all('p'):
		texts = p.get_text().split("：")
		if len(texts) == 1:
			continue
		for t in texts:
			fts = re.split('[一二三四五六七八九十]、', t)
			finalTexts.extend(fts)

	holidays = {}
	for i in range(0, len(finalTexts)):
		tobj = re.match('([0-9]*)年([0-9]*)月([0-9]*)日.([0-9]*)年([0-9]*)月([0-9]*)日', finalTexts[i])
		if tobj:
			holidays[finalTexts[i - 1]] = '%04d-%02d-%02d to %04d-%02d-%02d' % (
				int(tobj.group(1)), int(tobj.group(2)), int(tobj.group(3)), int(tobj.group(4)), int(tobj.group(5)),
				int(tobj.group(6)))
			continue

		tobj = re.match('([0-9]*)年([0-9]*)月([0-9]*)日.([0-9]*)月([0-9]*)日', finalTexts[i])
		if tobj:
			holidays[finalTexts[i - 1]] = '%04d-%02d-%02d to %04d-%02d-%02d' % (
				int(tobj.group(1)), int(tobj.group(2)), int(tobj.group(3)), int(tobj.group(1)), int(tobj.group(4)),
				int(tobj.group(5)))
			continue

		tobj = re.match('([0-9]*)年([0-9]*)月([0-9]*)日.([0-9]*)日', finalTexts[i])
		if tobj:
			holidays[finalTexts[i - 1]] = '%04d-%02d-%02d to %04d-%02d-%02d' % (
				int(tobj.group(1)), int(tobj.group(2)), int(tobj.group(3)), int(tobj.group(1)), int(tobj.group(2)),
				int(tobj.group(4)))
			continue

		tobj = re.match('([0-9]*)月([0-9]*)日.([0-9]*)月([0-9]*)日', finalTexts[i])
		if tobj:
			holidays[finalTexts[i - 1]] = '%04d-%02d-%02d to %04d-%02d-%02d' % (
				int(year), int(tobj.group(1)), int(tobj.group(2)), int(year), int(tobj.group(3)), int(tobj.group(4)))
			continue

		tobj = re.match('([0-9]*)月([0-9]*)日.([0-9]*)日', finalTexts[i])
		if tobj:
			holidays[finalTexts[i - 1]] = '%04d-%02d-%02d to %04d-%02d-%02d' % (
				int(year), int(tobj.group(1)), int(tobj.group(2)), int(year), int(tobj.group(1)), int(tobj.group(3)))
			continue
		tobj = re.match('([0-9]*)年([0-9]*)月([0-9]*)日', finalTexts[i])
		if tobj:
			holidays[finalTexts[i - 1]] = '%04d-%02d-%02d' % (
				int(tobj.group(1)), int(tobj.group(2)), int(tobj.group(3)))
			continue
		tobj = re.match('([0-9]*)月([0-9]*)日', finalTexts[i])
		if tobj:
			holidays[finalTexts[i - 1]] = '%04d-%02d-%02d' % (int(year), int(tobj.group(1)), int(tobj.group(2)))
			continue
	holidayDos = []
	for k in holidays.keys():
		if holidays[k].__contains__('to'):
			str = holidays[k].replace(' ', '')
			strlist = str.split('to')
			dateList = getBetweenDay(strlist[0], strlist[1])
			for i in range(0, len(dateList)):
				# print(k, dateList[i])
				holidayDo = HolidayInfo(k, dateList[i], '1')
				holidayDos.append(holidayDo)
		else:
			holidayDo = HolidayInfo(k, holidays[k], '1')
			holidayDos.append(holidayDo)
	# print(k, ":", holidays[k])
	# for i in range(len(holidayDos)):
	# 	print("date:%s type:%s desc:%s" % (holidayDos[i].date, holidayDos[i].type, holidayDos[i].desc))
	return holidayDos


print('********************************************')


class HolidayInfo:
	'''
	假日信息数据库实体类
	'''
	desc = ''
	date = ''
	type = ''

	def __init__(self, desc, date, type):
		self.desc = desc
		self.date = date
		self.type = type


def downloadFile(url):
	bs = BeautifulSoup(getContent(url, "www.gov.cn", "http://sousuo.gov.cn/").decode('utf-8'), "html.parser")

	tobj = re.match('国务院办公厅关于(.*)年部分节假日安排的通知', bs.title.string)
	if tobj:
		holidaydos = commonHoliday(tobj.group(1), bs)
		return holidaydos
	else:
		print('Unknown file type! Send mail now!')  # TODO: 未识别的通知类型，发送邮件给维护人处理


def genQueryFilesUrl(start, end, cnt):
	return '%s&mintime=%s&maxtime=%s&n=%d' % (
		'http://sousuo.gov.cn/data?t=zhengce_gw&q=%s&timetype=timezd&searchfield=title&pcodeJiguan=%s&puborg=&pcodeYear=&pcodeNum=&filetype=&p=&sort=score' % (
			quote('假'.encode('utf-8')), quote('国办发明电'.encode('utf-8'))), start, end, cnt)


def queryFiles(start, end):
	print("queryInfoStartDate:%s queryInfoEndDate:%s" % (start, end))
	host = "sousuo.gov.cn"
	referer = "http://sousuo.gov.cn/a.htm?t=zhengce"
	result = getContent(genQueryFilesUrl(start, end,
	                                     json.loads(getContent(genQueryFilesUrl(start, end, 1), host, referer))[
		                                     'searchVO']['totalCount']), host, referer)
	return json.loads(result)['searchVO']['listVO']


def getBetweenDay(begin_date, end_date):
	"""
	获取日期区间内的所有日期列表
	"""
	date_list = []
	begin_date = datetime.strptime(begin_date, "%Y-%m-%d")
	end_date = datetime.strptime(end_date, "%Y-%m-%d")
	while begin_date <= end_date:
		date_str = begin_date.strftime("%Y-%m-%d")
		date_list.append(date_str)
		begin_date += timedelta(days=1)
	return date_list


def getYearSub(date, subYears):
	"""
	日期年份作减
	"""
	d = datetime.strptime(date, '%Y-%m-%d')
	return (d - relativedelta(years=subYears)).strftime('%Y-%m-%d')


def getBetweenYear(begin_year, end_year):
	begin_date = datetime.strptime(begin_year, "%Y-%m-%d")
	end_date = datetime.strptime(end_year, "%Y-%m-%d")
	return range(begin_date.year, end_date.year + 1)


def add_holidays_info(holidaysList):
	"""节假日信息入库
	遍历数组的几种方式
	list = ['html', 'js', 'css', 'python']
	# 方法1
	print '遍历列表方法1：'
	for i in list:
		print ("序号：%s   值：%s" % (list.index(i) + 1, i))
	print '\n遍历列表方法2：'
	# 方法2
	for i in range(len(list)):
		print ("序号：%s   值：%s" % (i + 1, list[i]))
	# 方法3
	print '\n遍历列表方法3：'
	for i, val in enumerate(list):
		print ("序号：%s   值：%s" % (i + 1, val))
	# 方法3
	print '\n遍历列表方法3 （设置遍历开始初始位置，只改变了起始序号）：'
	for i, val in enumerate(list, 2):
		print ("序号：%s   值：%s" % (i + 1, val))
	"""
	usersvalues = []  # ((2021-11-15,1,元旦))
	for i in range(len(holidaysList)):
		usersvalues.append((holidaysList[i].date, holidaysList[i].type, holidaysList[i].desc))  # 注意要用两个括号扩起来

	conn = connect(**config)
	cs = conn.cursor()  # 获取光标
	# 注意这里使用的是executemany而不是execute，下边有对executemany的详细说明
	cs.executemany('insert into calendar_info (date,type,`desc`) values(%s,%s,%s)', usersvalues)
	conn.commit()
	cs.close()
	conn.close()
	print('数据库操作完成OK')


def deleteByYear(yearList):
	conn = connect(**config)
	cs = conn.cursor()  # 获取光标
	# 注意这里使用的是executemany而不是execute，下边有对executemany的详细说明
	sqlMatchCondition = ''
	for y in range(len(yearList)):
		sqlMatchCondition += ("^" + str(yearList[y]) + ".*|")
	sqlMatchCondition = sqlMatchCondition[0:len(sqlMatchCondition) - 1]
	sql = "delete from calendar_info where date REGEXP('" + sqlMatchCondition + "')"
	print("计划清除要取的年份历史数据sql=== %s" % sql)
	delSqlResult = cs.execute(sql)
	print("删除结果:", delSqlResult)
	conn.commit()
	cs.close()
	conn.close()


def main():
	print(
		'说明：可以通过参数设置查询日期范围\n示例：\n***.py ---- 查询所有\n***.py 2020-01-01 ---- 查询 2020-01-01 至 今日 的数据\n***.py 2020-01-01 2021-01-01 ---- 查询 2020-01-01 至 2021-01-01 的数据\n')

	if len(sys.argv) == 3:
		start = sys.argv[1]
		end = sys.argv[2]
	elif len(sys.argv) == 2:
		start = sys.argv[1]
		end = time.strftime("%Y-%m-%d", time.localtime())
	else:
		start = "2020-01-01"
		end = "2022-01-01"

	yearList = getBetweenYear(start, end)
	# 1.重跑删除要跑的历史年份数据
	deleteByYear(yearList)
	start = getYearSub(start, 1)
	fileResult = queryFiles(start, end)
	for file in fileResult:
		holidaydos = downloadFile(file['url'])
		if not holidaydos is None and len(holidaydos) > 0:
			# 入库
			add_holidays_info(holidaydos)


if __name__ == '__main__':
	main()
